import sys,os,csv,fnmatch
from datetime import datetime
import time
if len(sys.argv)<2 or len(sys.argv)>2:
  print "Usage: python hearstDataPrep.py '<DataFilesDirectory>'"
  sys.exit(1)
valMap={}
# valMap is used to convert the categorical values in to numerical values
for i in xrange(0,10):
   valMap[str(i)]=i
for ctr,alphabetOrd in enumerate(xrange(ord('a'),ord('z')+1)):
   valMap[chr(alphabetOrd)]=ctr+1
# 0,U and Null stands for Unknown or Nomatch for most of the attributes
valMap['u']=0
valMap['']=0

def categoryToNum(category):
   numVal=0
   try:
      numVal=float(category)
   except ValueError,e:
      for ctr,alpha in enumerate(category):
         try:
            numVal+=(ctr+1)*valMap[alpha.lower()]
         except KeyError,e:
            pass
   return str(numVal)

def classToNum(cls):
    if cls=='Y':
        return '1'
    else:
        return '0'
    return '0'

fileList=os.listdir(sys.argv[1])
dataFiles=[]
for file in fileList:
    if fnmatch.fnmatch(file,'Model*csv') or (file=='validation.csv'):
        dataFiles.append(file)
del fileList
os.chdir(sys.argv[1])

newData=open('newModel.csv','w')
validationData=open('newValidation.csv','w')

for file in dataFiles:
    oldHeader=[]
    newHeader=[]
    csvfile=csv.reader(open(file),delimiter=',',quotechar='"')

    # read the header only from the first file
    if len(oldHeader)==0 or file=='validation.csv':
      oldHeader=csvfile.next()
      for item in oldHeader[2:len(oldHeader)+1]:
        newHeader.append(item)
      if file!='validation.csv':
         #putting the class labels in the end
         for item in oldHeader[0:2]:
            newHeader.append(item)
         newData.write(','.join(newHeader)+'\n')
      else:
         validationData.write(','.join(newHeader)+'\n')
      
    else:
      csvfile.next()
      
    for row in csvfile:
        newrow=[]
        for item in row[2:len(row)]:
            newrow.append(categoryToNum(item))
        #converting mailed_date to unix timestamp
        newrow.append(str(categoryToNum(time.mktime(datetime.strptime(row[len(row)-1],"%a, %m/%d/%y %H:%M %p").date().timetuple()))))
        if file!='validation.csv':
            #putting the class labels in the end
            for item in row[0:2]:
                newrow.append(classToNum(item))
            newData.write(','.join(newrow)+'\n')
        else:
          validationData.write(','.join(newrow)+'\n')


newData.close()
validationData.close()
          
